package com.ybox.crawler.process.processor.impl;

import cn.hutool.http.HtmlUtil;
import com.ybox.common.core.domain.R;
import com.ybox.common.core.exception.ServiceException;
import com.ybox.common.core.utils.StringUtils;
import com.ybox.common.core.utils.UrlSpiltUtils;
import com.ybox.common.core.utils.uuid.IdUtils;
import com.ybox.crawler.domain.core.parse.ParseRule;
import com.ybox.crawler.domain.enums.CrawlerEnum;
import com.ybox.crawler.helper.CrawlerHelper;
import com.ybox.crawler.process.processor.AbstractCrawlerPageProcessor;
import com.ybox.crawler.utils.ParseRuleUtils;
import com.ybox.system.api.domain.SysFile;
import com.ybox.system.api.service.RemoteFileService;
import lombok.extern.log4j.Log4j2;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.ybox.common.core.utils.file.FileUtils.fileToMultipartFile;


/**
 * @author 16554
 */
@Component("CsdnDocPageProcessor")
@Log4j2
public class CrawlerDocPageProcessor extends AbstractCrawlerPageProcessor {

    @Autowired
    private CrawlerHelper crawlerHelper;

    @Autowired
    private RemoteFileService remoteFileService;

    @Value("${crawler.tempPic}")
    private String tempPic;

    @Value("${crawler.tempFile}")
    private String tempFile;

    /**
     * 处理页面数据
     *
     * @param page
     */
    @Override
    public void handelPage(Page page) {
        long currentTimeMillis = System.currentTimeMillis();
        String handelType = crawlerHelper.getHandelType(page.getRequest());
        log.info("开始解析目标页数，url:{},handelType:{}", page.getUrl(), handelType);
        // 保存页面 handelType 为 Custom 的时候才进行保存页面
        if (handelType.equals(CrawlerEnum.HandelType.CUSTOM.name())) {
            savePage(page);
        }
        // 获取目标页的抓取规则
        List<ParseRule> targetParseRuleList = getCrawlerXpath().getTargetParseRuleList();
        // 抽取当前page对象的有效的数据
        targetParseRuleList = ParseRuleUtils.parseHtmlByRuleList(page.getHtml(), targetParseRuleList);
        if (null != targetParseRuleList && !targetParseRuleList.isEmpty()) {
            for (ParseRule parseRule : targetParseRuleList) {
                // 将数据添加到page中，交给后续的pipline处理
                log.info("添加数据字段到page中的field,url:{},handelType:{},field:{}", page.getUrl(), handelType, parseRule.getField());
                if ("labels".equals(parseRule.getField())) {
                    // 如果是labels,就组合内容，并以，分割
                    page.putField(parseRule.getField(), parseRule.joinContent(","));
                } else {
                    page.putField(parseRule.getField(), parseRule.getMergeContent());
                }
            }
        }

        log.info("解析目标也数据完成，url:{},handelType:{},耗时:{}", page.getUrl(), handelType, System.currentTimeMillis() - currentTimeMillis);
    }

    @Override
    public boolean isNeedHandelType(String handelType) {
        return CrawlerEnum.HandelType.FORWARD.name().equals(handelType) || CrawlerEnum.HandelType.CUSTOM.name().equals(handelType);
    }

    @Override
    public boolean isNeedDocumentType(String documentType) {
        return CrawlerEnum.DocumentType.PAGE.name().equals(documentType);
    }

    @Override
    public int getPriority() {
        return 120;
    }

    /**
     * 保存页面
     */
    public void savePage(Page page) {
        Html html = page.getHtml();
        String icon = "";
        String origin = "";
        Map<String,String> map = null;
        CrawlerEnum.CrawlerType crawlerType = getCrawlerType();
        if (crawlerType == CrawlerEnum.CrawlerType.JUEJIN) {
            map = saveJuejin(html);
            icon = "iconfont icon-juejin-logo";
            origin = "掘金";
        } else if (crawlerType == CrawlerEnum.CrawlerType.ZHIHU) {
            map = saveZhihu(html);
            icon = "iconfont icon-shejiaotubiao-46";
            origin = "知乎";
        } else if (crawlerType == CrawlerEnum.CrawlerType.WEIXIN) {
            map = saveWeixin(html);
            icon = "iconfont icon-weixin";
            origin = "微信";
        } else if (crawlerType == CrawlerEnum.CrawlerType.CSDN) {
            map = saveCsdn(html);
            icon = "iconfont icon-juejin-logo";
            origin = "CSDN";
        } else {

        }
        page.putField("targetUrl",map.get("targetUrl"));
        page.putField("origin",origin);
        page.putField("icon",icon);
        page.putField("images",map.get("images"));
    }

    public Map<String, String> getHtml(CrawlerEnum.CrawlerType crawlerType) {
        HashMap<String, String> htmlMap = new HashMap<>(4);
        if (crawlerType == CrawlerEnum.CrawlerType.JUEJIN) {
            htmlMap.put("startHtml", "<html><head>");
            htmlMap.put("endHtml", "</div></body></html>");
            htmlMap.put("css", "<meta charset=\"utf-8\" />" +
                    "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1, user-scalable=no, viewport-fit=cover\" />" +
                    "<link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/app.5d6de2a.css\" />" +
                    "<link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/default.2eed0bc.css\" />" +
                    " <link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/60.daee106.css\" />");
            htmlMap.put("endHead", "</head><body style=\"padding-left: 2rem;padding-right: 2rem;background-color: #fff;\">" +
                    "<div class=\"main-area article-area\">");
        } else if (crawlerType == CrawlerEnum.CrawlerType.ZHIHU) {
            htmlMap.put("startHtml", "<html lang=\"zh\" xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"zh\"><head>");
            htmlMap.put("endHtml", "</body></html>");
            htmlMap.put("css", "<meta charset=\"utf-8\" />" +
                    "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1, user-scalable=no, viewport-fit=cover\" />" +
                    "<link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/column.css\" />" +
                    " <link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/7632.css\" />");
            htmlMap.put("endHead", "</head><body\">");
        } else if (crawlerType == CrawlerEnum.CrawlerType.WEIXIN) {
            htmlMap.put("startHtml", "<html><head>");
            htmlMap.put("endHtml", "</body></html>");
            htmlMap.put("css", "<link  href=\"http://127.0.0.1:9000/ybox-0/css/wechat.css\" rel=\"stylesheet\" type=\"text/css\"/>");
            htmlMap.put("endHead", "</head><body id=\"activity-detail\" class=\"zh_CN mm_appmsg  appmsg_skin_default appmsg_style_default \"><div id=\"js_article\" class=\"rich_media\">  <div id=\"js_top_ad_area\" class=\"top_banner\"></div><div class=\"rich_media_inner\"><div id=\"page-content\" class=\"rich_media_area_primary\"> <div class=\"rich_media_area_primary_inner\">");
        } else if (crawlerType == CrawlerEnum.CrawlerType.CSDN) {
            htmlMap.put("startHtml", "<html><head>");
            htmlMap.put("endHtml", "</div></body></html>");
            htmlMap.put("css", "");
            htmlMap.put("endHead", "");
        }
        return htmlMap;
    }

    /**
     * 从map中提取封面图片
     * @param map
     * @return
     */
    public String getImages(Map<String, String> map){
        return map.entrySet().stream()
                .map(e -> StringUtils.defaultIfBlank(e.getValue(), ""))
                .filter(e-> !e.endsWith("svg"))
                .limit(3)
                .collect(Collectors.joining(","));
    }

    public Map<String, String> saveCsdn(Html html) {
        return null;
    }

    public Map<String, String> saveJuejin(Html html) {
        Document document = html.getDocument();

        Map<String, String> htmlPart = getHtml(CrawlerEnum.CrawlerType.JUEJIN);
        // 因为掘金HTML格式比较固定，所以可以写死

        Map<String, String> map = picHandle(document, "src", null);
        Elements content = document.getElementsByTag("article");
        Elements articleEnd = document.getElementsByClass("article-end");
        String contentHtml = content.toString();
        for (Map.Entry<String, String> entry : map.entrySet()) {
            contentHtml = contentHtml.replace(entry.getKey(), entry.getValue());
        }

        String images = getImages(map);

        // 移除掘金的文章收录
        Elements columnContainer = articleEnd.select("div[class=column-container]");
        columnContainer.remove();

        // 移除掘金的拓展
        Elements extensionBanner = articleEnd.select("div[class=extension-banner]");
        extensionBanner.remove();


        String resultHtml = htmlPart.get("startHtml") + htmlPart.get("css") + htmlPart.get("endHead") + contentHtml + articleEnd + htmlPart.get("endHtml");
        String targetUrl = saveFile(resultHtml);
        log.info("爬取文章完成,保存的文章地址:{}",targetUrl);

        Map<String, String> result = new HashMap<>();
        result.put("images",images);
        result.put("targetUrl",targetUrl);
        return result;
    }

    public Map<String, String> saveZhihu(Html html) {
        Document document = html.getDocument();

        Map<String, String> htmlPart = getHtml(CrawlerEnum.CrawlerType.ZHIHU);
        // 因为掘金HTML格式比较固定，所以可以写死
        Map<String, String> map = picHandle(document, "data-original", "src");
        Elements content = document.getElementsByClass("Post-Main Post-Main-Mobile");
        String contentHtml = content.toString();
        for (Map.Entry<String, String> entry : map.entrySet()) {
            contentHtml = contentHtml.replace(entry.getKey(), entry.getValue());
        }

        String images = getImages(map);

        // 将data-original属性改为src
        contentHtml = contentHtml.replace("data-original", "src");

        String resultHtml = htmlPart.get("startHtml") + htmlPart.get("css") + htmlPart.get("endHead") + contentHtml + htmlPart.get("endHtml");
        String targetUrl = saveFile(resultHtml);
        log.info("爬取文章完成,保存的文章地址:{}",targetUrl);
        Map<String, String> result = new HashMap<>();
        result.put("images",images);
        result.put("targetUrl",targetUrl);
        return result;
    }

    public Map<String, String> saveWeixin(Html html) {
        Document document = html.getDocument();
        Map<String, String> htmlPart = getHtml(CrawlerEnum.CrawlerType.WEIXIN);

        Elements meta = document.select("meta");
        if (!StringUtils.equals("微信公众平台", meta.get(16).attr("content"))) {
            throw new ServiceException();
        }

        // 因为公众号HTML格式比较固定，所以可以写死
        String viewPort = meta.get(4).toString();
        String charset = meta.get(1).toString();


        Map<String, String> map = picHandle(document, "data-src", null);

        Elements title = document.getElementsByClass("rich_media_title");
        Elements content = document.getElementsByClass("rich_media_content");
        Elements author = document.getElementsByClass("rich_media_meta_list");

        String contentHtml = HtmlUtil.unescape(content.toString());

        // 不知道为啥解析出来的HTML visible是hidden，所以这里替换一下
        contentHtml = contentHtml.replaceFirst("hidden", "visible");
        for (Map.Entry<String, String> entry : map.entrySet()) {
            contentHtml = contentHtml.replace(entry.getKey(), entry.getValue());
        }

        String images = getImages(map);

        contentHtml = contentHtml.replace("data-src", "src");

        String resultHtml = htmlPart.get("startHtml") + htmlPart.get("css") + charset + viewPort + htmlPart.get("endHead") + title + author + contentHtml + htmlPart.get("endHtml");

        String targetUrl = saveFile(resultHtml);
        log.info("爬取文章完成,保存的文章地址:{}",targetUrl);
        Map<String, String> result = new HashMap<>();
        result.put("images",images);
        result.put("targetUrl",targetUrl);
        return result;
    }

    public String saveFile(String resultHtml) {
        String id = IdUtils.fastUUID();
        String filename = tempFile + id + ".html";
        String finalUrl = "";
        // 判断获取到的数据不为空
        try {
            // 如果有值则进行保存
            File html = new File(filename);
            if (!html.exists()) {
                html.createNewFile();
            }
            FileOutputStream fileOutputStream = new FileOutputStream(html);
            byte[] bytes = new byte[512];
            // 格式化html
            Document doc = Jsoup.parse(resultHtml);
            doc.outputSettings().prettyPrint(true);
            String formattedHtml = doc.html();

            bytes = formattedHtml.getBytes();
            fileOutputStream.write(bytes);
            fileOutputStream.close();

            // 上传文件到minio
            MultipartFile multipartFile = fileToMultipartFile(html);
            R<SysFile> upload = remoteFileService.upload(multipartFile);
            finalUrl = upload.getData().getUrl();
            html.delete();
        } catch (FileNotFoundException e) {
            throw new RuntimeException(e);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return finalUrl;
    }

    /**
     * 获取html图片
     *
     * @param urlString
     * @param suffix
     * @return
     * @throws Exception
     */
    public String download(String urlString, String suffix) throws Exception {
        // 获取URL并构造URL
        URL url = new URL(urlString);
        // 打开URL连接
        URLConnection con = url.openConnection();
        // 定义输入流
        InputStream is = con.getInputStream();

        // 定义1K的数据缓冲
        byte[] bs = new byte[1024];
        // 读取到的数据长度
        int len;
        /**
         *
         * 设置输出的文件流并设置下载路径及下载图片名称
         */
        String id = IdUtils.fastUUID();
        String filename = tempPic + id + "." + suffix;

        // 创建临时文件
        File file = new File(filename);

        FileOutputStream os = new FileOutputStream(file, true);
        // 开始读取
        while ((len = is.read(bs)) != -1) {
            os.write(bs, 0, len);
        }
        // 下载完毕，关闭所有链接
        os.close();
        is.close();
        // 上传文件到minio
        MultipartFile multipartFile = fileToMultipartFile(file);
        R<SysFile> upload = remoteFileService.upload(multipartFile);
        String finalUrl = upload.getData().getUrl();
        // 删除临时文件
        file.delete();
        return finalUrl;
    }

    /**
     * @param document   要匹配的文档
     * @param hrefAttr   图片地址属性
     * @param removeAttr 要移除的属性
     * @return
     */
    public Map<String, String> picHandle(Document document, String hrefAttr, String removeAttr) {
        Elements elements = document.select("img");
        HashMap<String, String> map = new HashMap<>(32);
        String path = null;
        for (Element element : elements) {

            // 移除属性
            if (StringUtils.isNotEmpty(removeAttr)) {
                element.removeAttr(removeAttr);
            }

            String src = element.attr(hrefAttr);

            if (src != null && !"".equals(src)) {
                if (map.get(src) == null || "".equals(map.get(src))) {
                    // 如果是base64图片
                    if (src.startsWith("data:image/")) {
                        map.put(src, src);
                        continue;
                    }
                    // 如果时//lf3-cdn-tos.bytescm.com 要去除开头给的两个斜杠
                    if (src.startsWith("//lf3-cdn-tos.bytescm.com")){
                        src = "https:" + src;
                    }
                    try {
                        // 下载图片
                        if (UrlSpiltUtils.getUrlMap(src).isEmpty()) {
                            String[] split = src.split("/");
                            int length = split.length;
                            String suffix = split[length - 1].split("\\.")[1];
                            suffix = suffix.replace("?", "");
                            path = download(src, suffix);
                        } else {
                            path = download(src, UrlSpiltUtils.getUrlMap(src).get("wx_fmt"));
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    map.put(src, path);
                }
            }
        }
        return map;
    }
}
